﻿using UnityEngine;
using System.Collections;
using System;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using System.Net.Sockets;  //套接字的命名空间
using System.Net; //IPAddress的命名空间
using System.Threading;
using System.Text;
using System.Collections.Generic; //线程的命名空间

[RequireComponent(typeof(AudioSource))]
public class SoundNeuralNet : MonoBehaviour {
    public static short MAX_AUDIO_VALUE;

    public int audioWidth = 256;
    public int servPort = 6000;
    public string hostName = "localhost";//8pl021df87.vicp.fun
    public int hostPort = 6000;//45167

    public NeuralNet neuralNet;//丘脑（感觉，平衡）
    public NeuralNet neuralNet1;//左脑
    public NeuralNet neuralNet2;//右脑
    public NeuralNet neuralSmallNet1;//左小脑(运动)
    public NeuralNet neuralSmallNet2;//右小脑（运动）

    public Texture img;

    private AudioClip audioClip;
    private int SamplingRate = 8000;
    private int audioLen = 1;
    private int lastPos;

    private float audioWaitTime;
    private float[] audioInClientData;
    private int audioInClientLenght;
    private AudioClip audioOut;
    private AudioSource audioSrc;
    private float[] audioOutData;
    private int k, audioVolWidth;

    private int voln = 0, volpn = 0;
    private float[] audioClientData;
    private int audioClientOutLen;
    private int audioInRPos, audioInRPos2, audioInWPos, audioInWPos2;
    private float[] audioInData, audioInData2;
    private System.Random rand = new System.Random();

    private int count = 0;
    private float deltaTime = 0;
    private float y;
    private float x;
    private int escape = 0;


    private int audioOutPos = 0;
    private WebCamTexture webCamTexture;

    //1.1声明套接字
    Socket serverSocket = null;
    //3.2 创建用来专门作为监听来电等待工作的线程
    Thread listenThread = null;

    Hashtable clients = new Hashtable();

    public void Awake() {
        float time = Time.time;

        audioOut = AudioClip.Create("sayer", 3 * SamplingRate, 1, SamplingRate, false);

        audioOutData = new float[audioLen * SamplingRate];
        audioSrc = GetComponent<AudioSource>();
        audioSrc.rolloffMode = AudioRolloffMode.Linear;
        audioSrc.clip = audioOut;
        audioVolWidth = 16;
        MAX_AUDIO_VALUE = short.MaxValue;
        audioInData = new float[SamplingRate];
        audioInData2 = new float[SamplingRate];
        audioInClientData = new float[SamplingRate];
        Debug.Log("MAX_AUDIO_VALUE=" + MAX_AUDIO_VALUE);
        Application.runInBackground = true;
        time = Time.time - time;
        Debug.Log("SNN Awake  " + time + "s");
    }

    //就是一个无参委托
    //public delegate void ThreadStart();
    //带了一个object参数，只能使用object类型当参数
    // public delegate void ParameterizedThreadStart(object? obj);

    // int maxStackSize 设置线程最大栈空间
    //	使用Lambda表达式
    private GameObject webCam;
    float lastTime = 0;
    // Use this for initialization
    public void Start()
    {
        audioClip = Microphone.Start(null, true, 3, SamplingRate);

        if (hostName == "localhost" && hostPort == servPort)
        {

            neuralNet.run = ThreadNNUpdate;
            neuralNet.StartLoad();
            neuralNet1.run = ThreadNN1Update;
            neuralNet1.StartLoad();
            neuralNet2.run = ThreadNN2Update;
            neuralNet2.StartLoad();
            neuralSmallNet1.run = ThreadNNS1Update;
            neuralSmallNet1.StartLoad();
            neuralSmallNet2.run = ThreadNNS2Update;
            neuralSmallNet2.StartLoad();

        }
        Thread bgThread = new Thread(ThreadBackGround);
        bgThread.IsBackground = true;
        bgThread.Start();

        webCam = GameObject.Find("WebCam");
        

        //audioSrc.Play();

        Debug.Log("MAX_AUDIO_VALUE=" + MAX_AUDIO_VALUE + "  OUTWIDTH=" + audioVolWidth);


    }

    private void ThreadBackGround()
    {
        while (serverRuning)
        {
            try
            {
                if (NeuralNet.loadData == 0)
                {
                    if (serverSocket == null)
                    {
                        StartServer();
                    }
                    if (clientSocket == null || clientSocket.Connected == false)
                    {
                        if (!ConnectServerStart())
                        {
                            Thread.Sleep(3000);
                            continue;
                        }
                    }
                    checkOutputAudio();
                    checkToClientsMessageList();
                    checkToServerMessageList();
                }
            }
            catch (Exception ex)
            {
                Debug.LogError(ex.ToString());
            }
            Thread.Sleep(neuralNet.sleep);
        }
    }
    private void checkToServerMessageList()
    {
        for (int i = 0; i < msToServerList.Count; i++)
        {
            MemoryStream ms = null;
            lock (msToServerList)
            {
                ms = msToServerList.Dequeue();
            }
            byte[]buff= ms.ToArray();

            SendToServer(buff);
        }
    }

    private void checkToClientsMessageList()
    {
        
        for (int i = 0; i < msToClientsList.Count; i++)
        {
            MemoryStream ms = null;
            lock (msToClientsList)
            {
                ms = msToClientsList.Dequeue();
            }
            SendToClients(ms.ToArray());
        }

    }

    private void checkOutputAudio()
    {
        //发送音频到客户端
        //if (audioOutPos >= SamplingRate)
        //{

        //    SendToClient_AudioOut();

        //    audioOutPos = 0;
        //}
    }

    private void StartServer()
    {
        //1.2调用Socket()函数 用于通信的套接字
        serverSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); //第一个参数为寻找地址的方式,此时选定为IPV4的地址; 第二个参数为数据传输的方式，此时选择的是Stream传输(能够准确无误的将数据传输到)；第三个参数为执行的协议，此时选择的是TCP协议；                                                                                   //2.1 设置地址 IPaddress 在using System.Net;下此时需引入
        IPAddress address = IPAddress.Any;
        //2.2 设置地址和端口
        IPEndPoint endPoint = new IPEndPoint(address, servPort); //第一个参数为要设置的IP地址，第二参数为端口号
        try
        {

            //2.套接字绑定端口号和IP
            serverSocket.Bind(endPoint);
            //toolStripStatusLabel1.Text = "服务已开启！";
            Debug.Log("开启服务成功");
        }
        catch (Exception ex)
        {
            Debug.LogError("开启服务失败：" + ex.Message);//ex.Message为出现异常的消息
            return;
        }
        //3.1监听套接字，等待
        serverSocket.Listen(3000); //参数为最大监听的用户数
        listenThread = new Thread(ListenConnectSocket);
        listenThread.IsBackground = true; //关闭后台线程
        listenThread.Start();



    }
    //3.3 用于判断用户是否链到服务器
    private bool serverRuning = true;

    //3.4 监听用户来电 等待
    void ListenConnectSocket()
    {
        while (serverRuning)
        {
            try
            {
                Socket clientSocket = serverSocket.Accept();
                string client = clientSocket.RemoteEndPoint.ToString();
                Debug.Log("客户连接: " + client);
                
                Thread thr = new Thread(ReceiveClientMsg);
                thr.IsBackground = true;
                thr.Start(clientSocket);
            }
            catch (Exception ex)
            {
                Debug.LogError(ex.ToString());
                Thread.Sleep(3000);
                //listenThread.Abort(ex.Message);
            }
        }
        //serverSocket.Close();
    }

    /// <summary>
    /// 服务器解释用户消息
    /// </summary>
    /// <param name="clientSocket"></param>
    private void ReceiveClientMsg(object clientSocket)
    {
        lock (clients)
        {
            clients.Add(clientSocket, true);
        }
        Socket client = clientSocket as Socket;
        byte[] recBuffer = new byte[1024 * 1024 * 2];
        MemoryStream ms = new MemoryStream(recBuffer, 0, recBuffer.Length);
        BinaryReader br = new BinaryReader(ms);
        int length = 0, len = 0, blen = 0;
        while (serverRuning)
        {
            try
            {
                do
                {
                    br.BaseStream.Position = 0;
                    ms.Position = 0;
                    len = client.Receive(recBuffer, length, recBuffer.Length - length, SocketFlags.None);
                    length += len;
                } while (length > 4 && (blen = br.ReadInt32()) > length);
                ms.SetLength(blen);
                br.BaseStream.SetLength(blen);
            }
            catch (Exception ex)
            {
                Debug.LogWarning(ex.ToString());
                break;
            }
            if (len == 0)
            {
                string str = client.RemoteEndPoint.ToString();
                Debug.Log(str+":下线了！");
                break;
            }
            else
            {
                //string msg = Encoding.Default.GetString(recBuffer, 0, length);
                //string msgStr = $"{DateTime.Now}【接收{client.RemoteEndPoint.ToString()}】{length}bytes";
                //Debug.Log(msgStr);

                if (blen > 4)
                {
                    try
                    {
                        HandleServerClientMsg(client, br);
                    }
                    catch (Exception ex)
                    {
                        //string str = client.RemoteEndPoint.ToString();
                        Debug.LogError(ex.ToString());
                        break;
                    }
                }
                else
                {
                    blen = 4;
                }
                for (int i = 0; i < length - blen; i++)
                {
                    recBuffer[i] = recBuffer[blen + i];
                }
                length -= blen;

            }
        }
        client.Close();
        lock (clients)
        {
            clients.Remove(client);
        }
    }

    private void HandleServerClientMsg(Socket s, BinaryReader r)
    {
        int id = r.ReadInt16();

        //Debug.Log("【消息】" + s.RemoteEndPoint.ToString() + ": " + id + " - " + r.BaseStream.Length);

        switch (id)
        {
            case 1000:
                HandleInputMsg(r);
                break;
            case 1001:
                HandleAudioMsg(r);
                break;
            case 1002:
                HandleTextureMsg(r);
                break;
            default:
                Debug.LogError(s.RemoteEndPoint.ToString()+":未知消息编号 "+id);
                s.Close();
                break;
        }
    }

    private void HandleTextureMsg(BinaryReader r)
    {
        int t = r.ReadByte();//0丘脑 1左脑 2右脑
        int x = r.ReadInt16();
        int y = r.ReadInt16();
        int xlen = r.ReadInt16();
        int ylen = r.ReadInt16();
        for (int i = 0; i < xlen; i++)
        {
            for (int j = 0; j < ylen; j++)
            {
                short v = r.ReadInt16();
                switch (t)
                {
                    case 0:

                        neuralNet.inputValue(x + i, y + j, v);
                        break;

                    case 1:

                        neuralNet1.inputValue(x + i, y + j, v);
                        break;

                    case 2:

                        neuralNet2.inputValue(x + i, y + j, v);
                        break;
                }
            }
        }
    }

    private void HandleInputMsg(BinaryReader r)
    {
        int t = r.ReadByte();//0丘脑 1左脑 2右脑

        for (int i = 0; r.BaseStream.Position + 6 < r.BaseStream.Length; i++)
        {
            int x = r.ReadInt16();
            int y = r.ReadInt16();
            short v = r.ReadInt16();
            switch (t)
            {
                case 0:
                    neuralNet.inputValue(x, y, v);
                    break;
                case 1:
                    neuralNet1.inputValue(x, y, v);
                    break;
                case 2:
                    neuralNet2.inputValue(x, y, v);
                    break;
                default:
                    break;
            }
        }
    }

    private void HandleAudioMsg(BinaryReader r)
    {
        int t = r.ReadByte();
        if (t == 1)//左脑
        {
            int len = r.ReadInt16();
            for (int i = 0; i < len; i++)
            {
                audioInData[audioInWPos++] = r.ReadSingle();
                if(audioInWPos == audioInData.Length)
                {
                    audioInWPos = 0;
                }
            }
        }
        if (t == 2)//右脑
        {
            int len = r.ReadInt16();
            for (int i = 0; i < len; i++)
            {
                audioInData2[audioInWPos2++] = r.ReadSingle();
                if (audioInWPos2 == audioInData2.Length)
                {
                    audioInWPos2 = 0;
                }
            }
        }
    }

    //1.1创建套接字
    Socket clientSocket = null;
    Thread clientThread = null;
    IPHostEntry hostAddr = null;
    //连接服务器
    private bool ConnectHostStart(string hostName, int port)
    {
        //1.2
        clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);//地址类型，数据传输方式，协议
        try
        {
            if (hostAddr == null)
            {
                hostAddr = Dns.GetHostEntry(hostName);
            }
            for (int i = hostAddr.AddressList.Length-1; i >=0 ; i--)
            {
                Debug.Log("与服务器"+hostAddr.AddressList[i].ToString()+":"+port+"连接中...");
                //3与服务器建立连接
                IPAddress address = hostAddr.AddressList[i];
                //2.2 设置IP地址和端口号
                IPEndPoint endPoint = new IPEndPoint(address, port);
                try
                {
                    clientSocket.Connect(endPoint); // 与服务器连接
                    Debug.Log("与服务器连接成功！");
                    //4.接收或发送消息 使用线程来实现
                    clientThread = new Thread(ReceiveMsg);
                    clientThread.IsBackground = true; //开启后台线程
                    clientThread.Start();
                    return true;
                }
                catch (Exception ex)
                {
                    Debug.Log("连接失败：" + ex.Message);
                }
            }
        }
        catch (Exception ex)
        {
            Debug.Log("连接失败：" + ex.Message);
        }
        return false;
    }
    //连接服务器
    private bool ConnectIpStart(string ipAddress, int port)
    {
        //1.2
        clientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);//地址类型，数据传输方式，协议

        Debug.Log("与服务器"+ipAddress+":"+port+"连接中...");
        //3与服务器建立连接
        //2.2 设置IP地址和端口号
        IPAddress address = IPAddress.Parse(ipAddress);
        IPEndPoint endPoint = new IPEndPoint(address, port);
        try
        {
            clientSocket.Connect(endPoint); // 与服务器连接
            Debug.Log("与服务器连接成功！");
            //4.接收或发送消息 使用线程来实现
            clientThread = new Thread(ReceiveMsg);
            clientThread.IsBackground = true; //开启后台线程
            clientThread.Start();
            return true;
        }
        catch (Exception ex)
        {
            Debug.Log("连接失败：" + ex.Message);
        }
        return false;
    }

    private void ReceiveMsg()
    {
        byte[] recBuffer = new byte[1024 * 1024 * 2];//声明最大字符内存
        MemoryStream ms = new MemoryStream(recBuffer, 0, recBuffer.Length);
        BinaryReader br = new BinaryReader(ms);
        int length = 0, len = 0, blen = 0;
        while (serverRuning)
        {
            try
            {
                do
                {
                    br.BaseStream.Position = 0;
                    ms.Position = 0;
                    len = clientSocket.Receive(recBuffer, length, recBuffer.Length - length, SocketFlags.None);
                    length += len;
                } while (length > 4 && (blen = br.ReadInt32()) > length);
                ms.SetLength(blen);
                br.BaseStream.SetLength(blen);
            }
            catch (SocketException ex)
            {
                Debug.LogWarning("Socket异常：" + ex.Message);
                break;
            }
            catch (Exception ex)
            {
                Debug.LogWarning("与服务器断开连接："+ex.ToString());
                break;
            }
            if (len <= 0)
            {
                Debug.LogWarning("与服务器断开连接："+len);
                break;
            }
            //接收到消息
            if (blen > 4)
            {
                //string msg = Encoding.Default.GetString(recBuffer, 0, length);//转译字符串(字符串，开始的索引，字符串长度)
                //string str = $"{DateTime.Now}【接收】{length}bytes";//接收的时间，内容，换行
                //Debug.Log(str);//添加到文本
                try
                {
                    HandleClientRevMsg(br);
                }
                catch (Exception ex)
                {
                    Debug.LogError("客户端异常：" + blen + ", " + ex.ToString());
                    break;
                }
            }
            else
            {
                blen = 4;
            }
            for (int i = 0; i < length - blen; i++)
            {
                recBuffer[i] = recBuffer[blen + i];
            }
            length -= blen;
            if (length < 0)
            {
                length = 0;
            }
        }
        clientSocket.Close();
    }

    private void HandleClientRevMsg(BinaryReader br)
    {

        int id = br.ReadInt16();
        //Debug.Log("【消息】"+id+" - "+br.BaseStream.Length);

        switch (id)
        {
            case 1000:// 神经元兴奋输出
                int t = br.ReadByte();//1左脑 2右脑
                //Debug.Log("兴奋输出 "+t+"  "+r.BaseStream.Length);
                short leftn = 0, rightn = 0, topn = 0, lown = 0;
                for (int i = 0; br.BaseStream.Position+6 < br.BaseStream.Length; i++)
                {
                    int ox = br.ReadInt16();
                    int oy = br.ReadInt16();
                    int ov = br.ReadInt16();
                    //Debug.Log("兴奋输出 " + t+"  ox"+ox+", oy"+oy+", v"+ov);
                    if (t == 1)
                    {
                        if (ox % 2 == 0)
                        {
                            y--;
                            topn+=100;
                        }
                        else
                        {
                            x--;
                            leftn+=100;
                        }

                    }
                    if (t == 2)
                    {
                        if (ox % 2 == 0)
                        {
                            y++;
                            lown+=100;
                        }
                        else
                        {
                            x++;
                            rightn+=100;
                        }
                    }
                }
                MemoryStream mms = new MemoryStream();
                BinaryWriter mw = new BinaryWriter(mms);
                mw.Write(0);
                mw.Write((short)1000);
                mw.Write((byte)0);//0丘脑神经
                //左
                mw.Write((short)0);
                mw.Write((short)1);
                mw.Write((leftn));
                //右
                mw.Write((short)1);
                mw.Write((short)1);
                mw.Write((rightn));
                //前
                mw.Write((short)2);
                mw.Write((short)1);
                mw.Write((topn));
                //后
                mw.Write((short)3);
                mw.Write((short)1);
                mw.Write(lown);

                long pos = mw.BaseStream.Position;
                mw.BaseStream.Position = 0;
                mw.Write((int)pos);
                mw.BaseStream.Position = pos;
                SendToServer(mms);
                //Debug.Log($"{DateTime.Now}【发送】丘脑感觉移动 {mms.Length}bytes");
                break;
            case 1001://声音输出
                int len = br.ReadInt16();
                float [] audioData = new float[len];
                for (int i = 0; i < len; i++)
                {
                    audioData[i] = br.ReadSingle();
                }
                audioClientData = audioData;
                audioClientOutLen = len;
                //Debug.Log("输出AI音频  "+audioClientOutLen);
                break;
            default:
                break;
        }
        
    }

    //发送消息
    private void SendToServer_Audio()
    {
        MemoryStream stream = new MemoryStream();
        BinaryWriter w = new BinaryWriter(stream);
        w.Write(0);
        w.Write((short)1001);
        w.Write((byte)1);//左脑
        w.Write((short)audioInClientLenght);
        for (int i = 0; i < audioInClientLenght; i++)
        {
            w.Write(audioInClientData[i]);
            //Debug.Log("audioInClientData: ["+i+"] " + audioInClientData[i]);
        }
        //Debug.Log($"{DateTime.Now}【发送】{stream.Length}bytes");
        long pos = w.BaseStream.Position;
        w.BaseStream.Position = 0;
        w.Write((int)pos);
        w.BaseStream.Position = pos;
        
        SendToServer(stream);

        stream = new MemoryStream();
        w = new BinaryWriter(stream);
        w.Write(0);
        w.Write((short)1001);
        w.Write((byte)2);//右脑
        w.Write((short)audioInClientLenght);
        for (int i = 0; i < audioInClientLenght; i++)
        {
            w.Write(audioInClientData[i]);
        }

        //Debug.Log($"{DateTime.Now}【发送】{stream.Length}bytes");
        pos = w.BaseStream.Position;
        w.BaseStream.Position = 0;
        w.Write((int)pos);
        w.BaseStream.Position = pos;
        
        SendToServer(stream);

       
    }

    private bool ConnectServerStart()
    {
        if (hostName != "localhost")
        {
            return ConnectHostStart(hostName, hostPort);
        }
        else
        {
            return ConnectIpStart("127.0.0.1", hostPort);
        }
    }

    Queue<MemoryStream> msToClientsList = new Queue<MemoryStream>();
    void SendToClients(MemoryStream ms)
    {
        lock (msToClientsList)
        {
            msToClientsList.Enqueue(ms);
        }

    }

    Queue<MemoryStream> msToServerList = new Queue<MemoryStream>();
    void SendToServer(MemoryStream ms)
    {
        if (clientSocket != null && clientSocket.Connected)
        {
            lock (msToServerList)
            {
                msToServerList.Enqueue(ms);
            }
        }
    }
    void SendToServer(byte[] buffer)
    {
        if (clientSocket != null && clientSocket.Connected)
        {
            try
            {
                int n = 0,sn=0;
                do
                {
                    n = clientSocket.Send(buffer, sn, buffer.Length - sn, SocketFlags.None);
                    if (n > 0 )
                    {
                        sn += n;
                        if (sn < buffer.Length)
                        {
                            Debug.LogError("发送数据不完整：" + sn + "/" + buffer.Length);
                        }
                    }
                    else
                    {
                        Debug.LogWarning("发送失败: "+n);
                        clientSocket.Close();
                        break;
                    }

                } while (n > 0 && buffer.Length > sn);
            }
            catch (Exception ex)
            {
                Debug.LogWarning(ex.Message);
            }
        }
    }
    void SendToClients(byte[] buffer)
    {
        lock (clients)
        {
            foreach (Socket client in clients.Keys)
            {
                try
                {
                    int n = 0, sn = 0;
                    do
                    {
                        n = client.Send(buffer, sn, buffer.Length - sn, SocketFlags.None);
                        if (n > 0)
                        {
                            sn += n;
                            if (sn < buffer.Length)
                            {
                                Debug.Log(client.RemoteEndPoint.ToString() + "发送数据不完整：" + sn + "/" + buffer.Length);
                            }
                        }
                        else
                        {
                            Debug.LogWarning(client.RemoteEndPoint.ToString()+" 发送失败: " + n);
                            break;
                        }
                    } while (n > 0 && buffer.Length > sn);
                    //Debug.Log($"{DateTime.Now}【发送】{client.RemoteEndPoint.ToString()}:{buffer.Length}bytes");
                }
                catch (Exception ex)
                {
                    Debug.LogWarning(ex.ToString());
                }
            }
        }
    }

    void SendToClient_AudioOut()
    {
        if (audioOutPos == 0)
            return;
        if (audioOutPos > audioOutData.Length)
        {
            audioOutPos = audioOutData.Length;
        }
        MemoryStream stream = new MemoryStream();
        BinaryWriter outw = new BinaryWriter(stream);
        outw.Write(0);
        outw.Write((short)1001);
        outw.Write((short)audioOutPos);
        for (int i = 0; i < audioOutPos; i++)
        {
            outw.Write(audioOutData[i]);
            audioOutData[i] = 0;
        }
        
        //Debug.Log("发送AI音频  " + audioOutPos);

        long pos = outw.BaseStream.Position;
        outw.BaseStream.Position = 0;
        outw.Write((int)pos);
        outw.BaseStream.Position = pos;
        SendToClients(stream);

       

    }

    


    // Update is called once per frame o verride

    public void Update()
    {
        deltaTime = Time.deltaTime;
        if (webCam != null)
        {
            Renderer renderer = webCam.GetComponent<Renderer>();
            webCamTexture = (WebCamTexture)renderer.material.mainTexture;
            // 看世界
            if (WebCam.hasWebCam && webCamTexture != null)
            {
                MemoryStream ms1 = new MemoryStream();
                BinaryWriter bw1 = new BinaryWriter(ms1);
                bw1.Write(0);
                bw1.Write((short)1000);
                bw1.Write((byte)1);
                MemoryStream ms2 = new MemoryStream();
                BinaryWriter bw2 = new BinaryWriter(ms2);
                bw2.Write(0);
                bw2.Write((short)1000);
                bw2.Write((byte)2);
                int ilen = neuralNet1.inputNum-(neuralNet.outputNum / 2 + audioWidth);
                for (int i = 0; i < ilen; i++)
                {
                    for (int j = 0; j < neuralNet1.inputHeight; j++)
                    {
                        int wx = (webCamTexture.width - ilen) / 2 + i, wy = (webCamTexture.height - neuralNet1.inputHeight) / 2 + j;
                        if (wx >= 0 && wx < webCamTexture.width && wy >= 0 && wy < webCamTexture.height)
                        {
                            Color color = webCamTexture.GetPixel(wx, wy);
                            short v = 0;
                            v += (short)((short)(color.r * 32) << 10);
                            v += (short)((short)(color.g * 32) << 5);
                            v += (short)(color.b * 32);
                            //if (i < neuralNet1.inputNum && j < neuralNet1.inputHeight)
                            //{
                            //    neuralNet1.inputValue(neuralNet1.inputNum / 2 + audioWidth + i, j, v);
                            //}
                            //if (i < neuralNet2.inputNum && j < neuralNet2.inputHeight)
                            //{
                            //    neuralNet2.inputValue(neuralNet2.inputNum / 2 + audioWidth + i, j, v);
                            //}
                            bw1.Write((short)neuralNet.outputNum / 2 + audioWidth + i);
                            bw1.Write((short)j);
                            bw1.Write((short)v);

                            bw2.Write((short)neuralNet.outputNum / 2 + audioWidth + i);
                            bw2.Write((short)j);
                            bw2.Write((short)v);
                        }
                    }
                }

                long pos = bw1.BaseStream.Position;
                bw1.BaseStream.Position = 0;
                bw1.Write((int)pos);
                bw1.BaseStream.Position = pos;

                pos = bw2.BaseStream.Position;
                bw2.BaseStream.Position = 0;
                bw2.Write((int)pos);
                bw2.BaseStream.Position = pos;

                SendToServer(ms1);
                SendToServer(ms2);

            }
        }
        
        
        
        //Debug.Log("audio-vol:" + deltaTime+"vol:"+vol);
        //Debug.Log ("audio-vol: ("+(nodeSay.thresholed?"1":"0")+(nodeSay1.thresholed?"1":"0")+(nodeSay2.thresholed?"1":"0")+(nodeSay3.thresholed?"1":"0")+(nodeSay4.thresholed?"1":"0")+(nodeSay5.thresholed?"1":"0")+(nodeSay6.thresholed?"1":"0")+(nodeSay7.thresholed?"1":"0")+") "+ vol+"  maxv="+maxv);

        // 录制麦克风,听世界的声音
        //int j = (int)(SamplingRate * deltaTime);
        //录音的长度，单位为秒，ui上可能需要显示
        if (Microphone.IsRecording(null))
        {
            //录音小于10秒
            int pos = Microphone.GetPosition(null);
            //audioLength = 1f * lastPos / SamplingRate;//录音时长
            audioWaitTime += deltaTime;
            if (pos > 0 && audioWaitTime >= audioLen)
            {
                if (pos < lastPos)
                {
                    lastPos = 0;
                }
                audioWaitTime = 0;

                audioInClientLenght = pos - lastPos;
                if (audioInClientLenght > 0)
                {
                    if (audioInClientLenght > audioInClientData.Length)
                    {
                        audioInClientLenght = audioInClientData.Length;
                    }
                    //Debug.Log("Audio GetData " + lastPos + "-" + pos);
                    if (audioClip.GetData(audioInClientData, lastPos))
                    {
                        //Debug.Log("录音长度：" + audioInClientLenght + "/" + audioClip.length);

                        SendToServer_Audio();
                    }
                    else
                    {
                        //Debug.Log("获取录音失败：" + audioInClientLenght);
                    }
                }

                lastPos = pos;

            }
            //Debug.Log("audio-in=" + audioLength + "s, audio-count:" + lastPos);
            //else
            //{
            //    data = null;
            //}
        }
        else
        {
            audioInClientData = null;
            //Debug.Log("audio-in: no!");
        }

        //播放声音，向世界发声音
        if (audioClientData !=null)
        {
            audioSrc.Stop();
            //Debug.Log("audio-say:" + audioOutPos + "/" + audioData.Length);
            audioOut.SetData(audioClientData, 0);
            audioClientOutLen = 0;
            audioClientData = null;
            audioSrc.Play();
            //k = 0;

            //for (int i = 0; i < audioClientData.Length; i++)
            //{
            //    audioClientData[i] = 0;
            //}
        }
        else
        {
            // Debug.Log("audio-say:" + audioOutPos);
        }

        if (Application.platform == RuntimePlatform.Android && Input.GetKeyDown(KeyCode.Escape)) // 返回键
        {
            //Code
            if (escape == 2)
            {
                Application.Quit();
            }
            else
            {
                escape++;
            }
        }
        if (Application.platform == RuntimePlatform.Android && Input.GetKeyDown(KeyCode.Home)) // Home键
        {
            //Code
        }


    }

    public void ThreadNN1Update()
    {
        int len = 0;
        // 听世界的声音
        if (audioInData != null && audioInRPos != audioInWPos)
        {
            short value = (short)(MAX_AUDIO_VALUE * audioInData[audioInRPos++]);
            if (audioInRPos == audioInData.Length)
            {
                audioInRPos = 0;
            }
            if (value < 0)
            {
                value = (short)(value * -1);
            }
            len = audioWidth / 2;
            for (int i = 0; i < len; i++)
            {
                for (int z = 0; z < neuralNet1.inputHeight; z++)
                {
                    neuralNet1.inputValue(neuralNet.outputNum / 2 + i, z, (short)(value * ((i + 1f) / len)));
                    neuralNet1.inputValue(neuralNet.outputNum / 2 + len + i, z, (short)(value * (1f * (len - i) / len)));
                }
            }
            //Debug.Log("audio-in-value="+ audioInPos +":"+ value); 
        }
        // 左脑到右脑
        for (int i = 1; i <= neuralNet1.hideLayNum; i++)
        {
            for (int z = 0; z < neuralNet1.hideHeight; z++)
            {
                neuralNet2.inputValue(i, 0, z, neuralNet1.getInputLayer(i, neuralNet1.hideWidth - 1, z));
            }
            
        }
        // 左脑到左小脑
        for (int i = 0; i < neuralNet1.outputNum; i++)
        {
            for (int z = 0; z < neuralNet1.outputHeight; z++)
            {
                neuralSmallNet1.inputValue(i,z, neuralNet1.getOutInputLayer(i,z));
            }
        }
    }

    public void ThreadNN2Update()
    {
        int len = 0;
        if (audioInData2 != null && audioInRPos2 != audioInWPos2)
        {
            short value = (short)(MAX_AUDIO_VALUE * audioInData2[audioInRPos2++]);
            if (audioInRPos2 == audioInData2.Length)
            {
                audioInRPos2 = 0;
            }
            if (value < 0)
            {
                value = (short)(value * -1);
            }
            len = audioWidth / 2;
            for (int i = 0; i < len; i++)
            {
                for (int z = 0; z < neuralNet2.inputHeight; z++)
                {
                    neuralNet2.inputValue(neuralNet.outputNum / 2 + i, z, (short)(value * ((i + 1f) / len)));
                    neuralNet2.inputValue(neuralNet.outputNum / 2 + len + i, z, (short)(value * (1f * (len - i) / len)));
                }

            }
            //Debug.Log("audio-in-value="+ audioInPos +":"+ value); 
        }

        // 右脑到左脑
        for (int i = 1; i <= neuralNet2.hideLayNum; i++)
        {
            for (int z = 0; z < neuralNet2.hideHeight; z++)
            {
                neuralNet1.inputValue(i, 0, z, neuralNet2.getInputLayer(i, neuralNet2.hideWidth - 1, z));
            }
        }
        // 右脑到右小脑
        for (int i = 0; i < neuralNet2.outputNum; i++)
        {
            for (int z = 0; z < neuralNet2.outputHeight; z++)
            {
                neuralSmallNet2.inputValue(i,z, neuralNet2.getOutInputLayer(i,z));
            }
        }
    }
    private float maxVol, minVol;
    public void ThreadNNS1Update()
    {
        int len = Math.Max(10, audioWidth) / 2;
        int count = 0;
        MemoryStream mstream = null;
        BinaryWriter outw = null;
        float voln = 0,volpn=0;

        for (int i = 0; i < neuralSmallNet1.outputNum; i++)
        {
            for (int z = 0; z < neuralSmallNet1.outputHeight; z++)
            {
                if (i < audioWidth)
                {
                    //lock (neuralNet)
                    {
                        if (i < len)
                        {
                            voln += neuralSmallNet1.checkOut(i, z)?1:0;
                        }
                        else
                        {
                            // neuralSmallNet1.clear(i);
                            volpn += neuralSmallNet1.checkOut(i, z) ? 1 : 0;
                            //neuralSmallNet1.clear(len + i);
                        }
                    }
                }
                else
                {
                    short v = neuralSmallNet1.getOutInputLayer(i, z);
                    if (v > 0)
                    {
                        if (count == 0)
                        {
                            mstream = new MemoryStream();
                            outw = new BinaryWriter(mstream);
                            outw.Write(0);
                            outw.Write((short)1000);
                            outw.Write((byte)1);//1左脑 2右脑
                        }
                        count++;
                        outw.Write((short)i);
                        outw.Write((short)z);
                        outw.Write(v);
                    }
                }
            }
        }
        if (count > 0)
        {
            long pos = outw.BaseStream.Position;
            outw.BaseStream.Position = 0;
            outw.Write((int)pos);
            outw.BaseStream.Position = pos;

            SendToClients(mstream);
        }
        if(audioOutPos<audioOutData.Length){
            float vol = voln + volpn;
            if (vol > maxVol)
            {
                maxVol = vol;
            }
            audioOutData[audioOutPos] = vol/maxVol;
            audioOutPos += (int)Math.Round(SamplingRate * neuralSmallNet1.sleep / 1000f);
            voln =0;volpn = 0;
            //Debug.Log((audioOutPos - 1) + " : " + audioOutData[audioOutPos - 1]);
        }

    }

    public void ThreadNNS2Update()
    {
        int len = Math.Max(10, audioWidth) / 2;
        int count = 0;
        MemoryStream mstream2 = null;
        BinaryWriter outw = null;
        float voln = 0,volpn=0;

        for (int i = 0; i < neuralSmallNet2.outputNum; i++)
        {
            for (int z = 0; z < neuralSmallNet2.outputHeight; z++)
            {
                if (i < audioWidth)
                {
                    //lock (neuralNet)
                    {
                        if (i < len)
                        {
                            voln += neuralSmallNet2.checkOut(i, z)?1:0;
                        }
                        else
                        {
                            //neuralSmallNet1.clear(i);
                            volpn += neuralSmallNet2.checkOut(i, z) ? 1 : 0;
                            //neuralSmallNet2.clear(len+i);
                        }
                    }
                }
                else
                {
                    short v = neuralSmallNet2.getOutInputLayer(i, z);
                    if (v > 0)
                    {
                        if (count == 0)
                        {
                            mstream2 = new MemoryStream();
                            outw = new BinaryWriter(mstream2);
                            outw.Write(0);
                            outw.Write((short)1000);
                            outw.Write((byte)2);//1左脑 2右脑
                        }
                        count++;
                        outw.Write((short)i);
                        outw.Write((short)z);
                        outw.Write(v);
                    }
                }
            }
        }
        if (count > 0)
        {
            long pos = outw.BaseStream.Position;
            outw.BaseStream.Position = 0;
            outw.Write((int)pos);
            outw.BaseStream.Position = pos;

            SendToClients(mstream2);
        }
        if(audioOutPos<audioOutData.Length){
            float vol = -(voln + volpn);
            if (vol < minVol)
            {
                minVol = vol;
            }
            audioOutData[audioOutPos] = -vol / minVol;
            audioOutPos += (int)Math.Round(SamplingRate * neuralSmallNet2.sleep / 1000f);
            voln = 0;volpn = 0;
            //Debug.Log((audioOutPos-1)+" : "+audioOutData[audioOutPos-1]);
        }
    }

    public void ThreadNNUpdate()
    {
        int maxNum = neuralNet.outputNum/2;

        //丘脑输入到左右脑
        for (int i = 0; i < maxNum; i++)
        {
            for (int z = 0; z < neuralNet.outputHeight; z++)
            {
                neuralNet1.inputValue(i,z, neuralNet.getOutInputLayer(i,z));
                neuralNet2.inputValue(i,z, neuralNet.getOutInputLayer(maxNum + i,z));
            }
            
        }

        // 感觉时钟
        DateTime now = DateTime.Now;
        neuralNet.inputValue(0,0, (short)now.Year);
        neuralNet.inputValue(1, 0, (short)now.Month);
        neuralNet.inputValue(2, 0, (short)now.Day);
        neuralNet.inputValue(3, 0, (short)now.Hour);
        neuralNet.inputValue(4, 0, (short)now.Minute);
        neuralNet.inputValue(5, 0, (short)now.Second);
        neuralNet.inputValue(6, 0, (short)now.Millisecond);

        // 输出声带音频数据
        //if (voln>0&&volpn > 0)
        // {
        //Debug.Log("SoundOutput  "+voln+","+volpn);

        //maxNum = Math.Min(neuralSmallNet1.outputNum, audioWidth) / 2 * neuralSmallNet1.outputHeight;
        //maxNum += Math.Min(neuralSmallNet2.outputNum, audioWidth) / 2 * neuralSmallNet2.outputHeight;
        //float vol = (1f*voln / maxNum);
        //Debug.Log("声带："+vol);
        //for (int i = 0; i < 67f * volpn / maxNum; i++)
        //{
        //    if (audioOutPos < audioOutData.Length)
        //    {
        //audioData [l] = (vol * rate) / MAX_AUDIO_VALUE;
        //        audioOutData[audioOutPos++] = vol;
        //Debug.Log("audio-out-Pos=" + audioOutPos + ":" + vol);
        //   }
        //if (l + 1 >= audioData.Length)
        //    break;
        //audioData [l + 1] = -(vol * rate) / MAX_AUDIO_VALUE;
        //    if (volpn%2==0&&audioOutPos < audioOutData.Length)
        //    {
        //       audioOutData[audioOutPos++] = -vol;
        //audioOutPos += (67 / (int)Math.Round(Math.Max(1, 67f * volpn / maxNum)))-2;
        //if (audioOutPos >= audioOutData.Length)
        // {
        //audioOutPos = audioOutData.Length;
        //      SendToClient_AudioOut();
        //      audioOutPos = 0;
        //  }
        //   }
        //}
        //  voln = 0; volpn = 0;
        //}
        //else
        //{
        if (/*audioOutPos > 0 && (audioOutData[audioOutPos - 1] == 0 ||*/ audioOutPos >= audioOutData.Length)
        {
            //Debug.Log("Send audio-pos:" + audioOutPos);
            SendToClient_AudioOut();
            audioOutPos = 0;
        }
        else
        {
            //Debug.Log("Write audio-pos:"+audioOutPos);
        }
            //audioOutPos += 66;
            //for (int i = 0; i < 66/2; i++)
            //{
            //    audioOutData[audioOutPos++] = rand.Next(1, 100) / 10000f;
            //    audioOutData[audioOutPos++] = -rand.Next(1, 100) / 10000f;
            //}
            //audioOutPos += 67;
       // }
        
    }

    void OnGUI()
    {
        if (img != null)
        {
            x = -Screen.width / 2f > x ? -Screen.width / 2f + img.width / 2 : Mathf.Min(Screen.width / 2 - img.width, x);
            y = -Screen.height / 2f > y ? -Screen.height / 2f+ img.height/2 : Mathf.Min(Screen.height / 2 - img.height, y);
            GUI.DrawTexture(new Rect((Screen.width - img.width) / 2f + x, (Screen.height - img.height) / 2f + y, img.width * 1f, img.height * 1f), img);
        }
        if (webCamTexture != null)
        {
            GUI.TextField(new Rect(10f, 10f, 500, 24), "WebCam: " + webCamTexture.width + "x" + webCamTexture.height + " " + WebCam.hasWebCam);
        }

        if (NeuralNet.loadData==0 && neuralNet.hideLayNum<200&&neuralNet.hideWidth<200)
        {
            int xn = neuralNet.hideLayNum+2, yn = neuralNet.hideWidth;
            for (int i = 0; i < xn; i++)
            {
                for (int j = 0; j < yn; j++)
                {
                    int xi = i ;
                    int yi = j ;
                    //for (int z = 0; z < neuralNet.outputHeight; z++)
                    //{
                        if (neuralNet.check(xi, yi, 0))
                        {
                            DrawCircle(new Rect((Screen.width - (neuralNet.hideLayNum + 2) * 10) / 2 + i * 10, (Screen.height - neuralNet.hideWidth * 10) / 2 + j * 10, 8, 8));
                        }
                        else
                        {
                            DrawCircle(new Rect((Screen.width - (neuralNet.hideLayNum + 2) * 10) / 2 + i * 10, (Screen.height - neuralNet.hideWidth * 10) / 2 + j * 10, 5, 5));
                        }
                    //}
                }
            }
            //int xn = Screen.width - 50 / 10, yn = Screen.height - 50 / 10;
            //for (int i = 0; i < xn; i++)
            //{
            //    for (int j = 0; j < yn; j++)
            //    {
            //        int xi = i * (int)((neuralNet.hideLayNum + 2f) / xn);
            //        int yi = j * (int)(1f * neuralNet.hideWidth / yn);

            //        if (neuralNet.check(xi, yi))
            //        {
            //            DrawCircle(new Rect(20 + i * 10, 20 + j * 10, 8, 8));
            //        }
            //        else
            //        {
            //            DrawCircle(new Rect(20 + i * 10, 20 + j * 10, 5, 5));
            //        }
            //    }
            //}
        }

        if (NeuralNet.loadData > 0)
        {
            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) / 2f + 150, 200, 24), (neuralNet.initing?"Init Data: ":"Load Data: ") + neuralNet.name + "  (" + neuralNet.count + "/" + neuralNet.hideLayNum + ") ");

            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) / 2f + 150 + 24, 200, 24), (neuralNet1.initing ? "Init Data: " : "Load Data: ") + neuralNet1.name + "  (" + neuralNet1.count + "/" + neuralNet1.hideLayNum + ") ");

            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) / 2f + 150 + 24 * 2, 200, 24), (neuralNet2.initing ? "Init Data: " : "Load Data: ") + neuralNet2.name + "  (" + neuralNet2.count + "/" + neuralNet2.hideLayNum + ") ");

            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) / 2f + 150 + 24 * 3, 200, 24), (neuralSmallNet1.initing ? "Init Data: " : "Load Data: ") + neuralSmallNet1.name + "  (" + neuralSmallNet1.count + "/" + neuralSmallNet1.hideLayNum + ") ");

            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) / 2f + 150 + 24 * 4, 200, 24), (neuralSmallNet2.initing ? "Init Data: " : "Load Data: ") + neuralSmallNet2.name + "  (" + neuralSmallNet2.count + "/" + neuralSmallNet2.hideLayNum + ") ");
        }

        if (clientSocket!=null&&clientSocket.Connected)
        {
            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) - 30, 200, 24), "连接服务器正常！");
        }
        else
        {
            GUI.Label(new Rect((Screen.width) / 2f - 100, (Screen.height) - 30, 200, 24), "与服务器连接断开！");
        }
    }

    private void DrawCircle(Rect rect)
    {
        GUI.DrawTexture(rect, img);
    }

    //void OnDrawGizmos()
    //{
    //    for (int i = 0; i < neuralNet.hideLayNum + 2; i++)
    //    {
    //        for (int j = 0; j < neuralNet.hideWidth; j++)
    //        {
    //            if (neuralNet.check(i, j))
    //            {
    //                DrawCircle(new Vector3(100 + i * 10, 100 + j * 10), 8, 8);
    //            }
    //            else
    //            {
    //                DrawCircle(new Vector3(100 + i * 10, 100 + j * 10), 1, 8);
    //            }
    //        }
    //    }
    //}

    //void DrawCircle(Vector3 center, float radius, int segments)
    //{
    //    float angleStep = 360f / segments;
    //    Vector3 previousPoint = center + Vector3.forward * radius;

    //    for (int i = 1; i <= segments; i++)
    //    {
    //        float angle = angleStep * i;
    //        Vector3 newPoint = center + new Vector3(Mathf.Cos(angle * Mathf.Deg2Rad) * radius, 0, Mathf.Sin(angle * Mathf.Deg2Rad) * radius);
    //        Gizmos.DrawLine(previousPoint, newPoint);
    //        previousPoint = newPoint;
    //    }
    //}



    void OnApplicationQuit()
    {
        serverRuning = false;
        neuralNet.OnDestroy();
        neuralNet1.OnDestroy();
        neuralNet2.OnDestroy();
        neuralSmallNet1.OnDestroy();
        neuralSmallNet2.OnDestroy();
        
        //PlayerPrefs.set
        if (Application.platform != RuntimePlatform.WindowsEditor)
            {
            if (NeuralNet.loadData == 0)
            {
                neuralNet.SaveData();
                neuralNet1.SaveData();
                neuralNet2.SaveData();
                neuralSmallNet1.SaveData();
                neuralSmallNet2.SaveData();
            }
        }

        neuralNet.OnApplicationQuit();
        neuralNet1.OnApplicationQuit();
        neuralNet2.OnApplicationQuit();
        neuralSmallNet1.OnApplicationQuit();
        neuralSmallNet2.OnApplicationQuit();
    }

    
}
